#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jul 22 20:25:59 2018
@project: 天池比赛-A股主板上市公司公告信息抽取
@group: MZH_314
@author: LHQ

@desc:
    对增减持的文本提取部分做分类算法的训练数据准备,主要包含对句子的切割
    及之后的根据赛方提供结构数据(train文件)对切割后的句子贴标签

"""
import re
import os

import pandas as pd

from reportIE.ner.alias import CompanyAlias
from reportIE.preprocess import ZjcReport
from reportIE.utils.load_data import load_report, load_reports, load_zengjianchi_train_data
from reportIE.ner.recognizer import make_recognizer
from reportIE.preprocess import make_converter


date_recognizer = make_recognizer('date')
stockquantity_recognizer = make_recognizer("stockquantity")
percent_recognizer = make_recognizer('percent')
amount_recognizer = make_recognizer('amount')

date_converter = make_converter("date")
percent_converter = make_converter("percent")



def process_texts(texts):
    """在提取之前，先对文本做些预处理
    """
    def rpl(m):
        s = m.group(1)
        s = re.sub("[,，]", "", s)
        return s
    new_texts = []
    for text in texts:
        # 句子分割
        for s in re.split("[。]", text):
            # 含逗号的数值去逗号
            s_new = re.sub("([\d,，\.]{4,})", rpl, s)
            # 万 单位转换
            s_new = re.sub("\s*万股", "0000股", s_new)
            # 日期转换
            for entity in date_recognizer.recognize(s_new):
                ent = entity.entity
                date = date_converter.convert(ent)
                s_new = re.sub(ent, date, s_new)
#            # 百分比转换成小数
#            for entity in percent_recognizer.recognize(s_new):
#                ent = entity.entity
#                percent = percent_converter.convert(s_new)
#                s_new = re.sub(ent, percent, s_new)
#                s_new = re.sub("%", "", s_new)
            if s_new.strip():
                new_texts.append(s_new)
    return new_texts


def label_change_event(df_std, contents):
    data = []
    indexes = set()
    sentences = set()
    std_data = []
    report_id = df_std['report_id'].values[0]
    for _, row in df_std.iterrows():
        shareholder_fullname = row['shareholder_fullname']
        shareholder_shortname = row['shareholder_shortname']
        changed_date = row['changed_date']
        changed_quantity = row['changed_quantity']
#        final_percent = row['final_percent']
        final_quantity = row['final_quantity']
        try:
            changed_quantity = int(changed_quantity)
        except ValueError:
            pass
        try:
            final_quantity = int(final_quantity)
        except ValueError:
            pass
        
        std_data.append(row.tolist())
        
        if isinstance(shareholder_shortname, str) and len(shareholder_shortname) > 0:
            p1 = re.compile("(?:%s|%s).*?%s" %(shareholder_fullname, shareholder_shortname, changed_quantity))
        else:
            p1 = re.compile("%s.*?%s" %(shareholder_fullname, changed_quantity))
        p2 = re.compile("%s.*?%s" % (changed_date, changed_quantity))
        p3 = re.compile("%s股"%final_quantity)
        p4 = re.compile("占.*?[\d\.]+%?")
        patterns = [p1, p2, p3, p4]
        
        
        for i, s in enumerate(new_contents):
            for p in patterns:
                m = p.search(s)
                if m:
                    sentences.add(s)
                    indexes.add(i)
                    
    for s in sentences:
        item = {"report_id": report_id,
                "origin_text": s,
                "std_data": std_data,
                "is_target": 1}
        data.append(item)
        
    for i, s in enumerate(new_contents):
        if i not in indexes:
            item = {"report_id": report_id,
                    "origin_text": s,
                    "std_data": [[]],
                    "is_target": 0}
            data.append(item)
    return data


if __name__ == "__main__":
    train_path = os.path.abspath("../data/[new] FDDC_announcements_round1_train_result_20180616/zengjianchi.train")
    df_train = load_zengjianchi_train_data(train_path)
    
#    report_id = "10164"    
#    html_path = "/home/lhq/project/FDDC2018/origin_data/增减持/html/%s.html" % report_id
#    _, html = load_report(html_path)
    
    html_dir = os.path.abspath("../data/round1_train_20180518/增减持/html")
    save_dir = os.path.abspath("../data/training_data")
    if not os.path.exists(save_dir):
        os.mkdir(save_dir)
    
    
    datas = []
    for report_id, html in load_reports(html_dir):
        report = ZjcReport(html, report_id)
        if not report.has_table():
            contents = report.flat_contents2
            
            df_std = df_train.query("report_id==@report_id")
            if df_std.empty:
                continue
            
            alias = CompanyAlias(contents)
            short_full_namemap = alias.get_allalias()
            
            new_contents = process_texts(contents)
            
            data = label_change_event(df_std, new_contents)
            datas.extend(data)

    df = pd.DataFrame(datas)
    
    save_path = os.path.join(save_dir, "zengjianchi_text.csv")
    df.to_csv(save_path, index=False)
    
    
